#coding=utf8
from __future__ import division
from torch import nn
import torch
import torch.utils.data as torchdata
from torchvision import datasets,transforms
import os,time
import pandas as pd
import numpy as np
from sklearn.metrics import f1_score, precision_recall_fscore_support, classification_report,confusion_matrix
from glob import glob
import cPickle
import itertools


val_true = np.load('val2/val-labels.npy')
le = cPickle.load(open('val2/label-encoder.pkl','rb'))


files = [
    '[old]resnet20-0.7454-aug8-0.7576.npy',
    '[old]resnet20-crop-smpunknown-0.7653.npy',
    '[old]xception_crop_smpunknown-0.7712-aug8-0.7857.npy',
    '[old]xception-0.7601.npy',
    '[old]resnet20-crop-smpunknown-0.7653-aug8-0.7747.npy',
    'resnet20_crop-0.7531-aug8-0.7589.npy',
    'resnet20_crop_NoAdpt_smpunknown-0.7741.npy',
    'resnet20_crop_smpunknown-0.7824.npy',
    'resnet20_crop_smpunknown_add[qso,galaxy]-0.7666-aug8-0.7698.npy',
    'resnet20_crop_smpunknown_add[qso]-0.7729-aug8-0.7838.npy',
    'xception_crop-0.7635-aug8-0.7769.npy',
    'xception_crop_smpunknown-0.7785-aug8-0.7855.npy'
         ]

# resnet20_redshift-0.8171-aug8-0.8142.npy
# SEresnet20_smpunknown-0.8252-aug8-0.8291.npy
# xception_crop_smpunknown-0.8231-aug8-0.8286.npy
# xception-0.8138.npy
# resnet20_crop_NoAdpt_smpunknown-0.8246-aug8-0.8280.npy



# [old]resnet20-0.7454-aug8-0.7576.npy
# [old]xception_crop_smpunknown-0.7712-aug8-0.7857.npy
# [old]xception-0.7601.npy
# resnet20_crop_NoAdpt_smpunknown-0.7741.npy
# resnet20_crop_smpunknown-0.7824.npy
# xception_crop-0.7635-aug8-0.7769.npy
# xception_crop_smpunknown-0.7785-aug8-0.7855.npy

bset_score = 0
best_comb = []
for comb_num in range(1,len(files)+1):
    for comb in itertools.combinations(files,comb_num):



        scores = []
        for file_name in comb:
            file_path = os.path.join('./val2',file_name)

            score = np.load(file_path)
            scores.append(score)

        scores_pred = np.mean(np.array(scores),axis=0)
        val_preds = np.argmax(scores_pred,axis=1)
        val_f1 = f1_score(val_true, val_preds,average='macro')

        # labels=['galaxy','qso','star','unknown']
        # cm = confusion_matrix(val_true, val_preds)
        #
        # cm = pd.DataFrame(cm)
        #
        # cm.index = ['galaxy','qso','star','unknown']
        # cm.columns = ['galaxy','qso','star','unknown']

        if val_f1 > bset_score:
            bset_score  = val_f1
            best_comb = comb

for item in best_comb:
    print item
print bset_score